{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "%load_ext tensorboard\n",
    "import datetime, os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = tf.keras.datasets.fashion_mnist\n",
    "\n",
    "(x_train_1, y_train), (x_test_1, y_test) = mnist.load_data()\n",
    "x_train_1, x_test_1 = x_train_1 / 255.0, x_test_1 / 255.0\n",
    "\n",
    "x_train = x_train_1.reshape(60000,28,28,1)\n",
    "x_test = x_test_1.reshape(10000,28,28,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_11\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_20 (Conv2D)           (None, 28, 28, 32)        320       \n",
      "_________________________________________________________________\n",
      "batch_normalization_12 (Batc (None, 28, 28, 32)        128       \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 28, 28, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_20 (MaxPooling (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_21 (Conv2D)           (None, 14, 14, 64)        18496     \n",
      "_________________________________________________________________\n",
      "batch_normalization_13 (Batc (None, 14, 14, 64)        256       \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_21 (MaxPooling (None, 7, 7, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_10 (Flatten)         (None, 3136)              0         \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 256)               803072    \n",
      "_________________________________________________________________\n",
      "batch_normalization_14 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 84)                21588     \n",
      "_________________________________________________________________\n",
      "batch_normalization_15 (Batc (None, 84)                336       \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 84)                0         \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 10)                850       \n",
      "=================================================================\n",
      "Total params: 846,070\n",
      "Trainable params: 845,198\n",
      "Non-trainable params: 872\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Conv2D, Dense, MaxPool2D, Dropout, Flatten, Reshape, BatchNormalization, Activation\n",
    "from tensorflow.keras.constraints import max_norm\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras import initializers\n",
    "from tensorflow.keras import activations\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(kernel_initializer=initializers.he_normal(seed=12),  \n",
    "                 bias_initializer=initializers.he_uniform(seed=12),\n",
    "    filters=32, kernel_size=(3,3), padding='same', activation='linear', input_shape=(28, 28, 1)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(activations.relu))\n",
    "model.add(MaxPool2D(padding='same'))\n",
    "model.add(Conv2D(kernel_initializer=initializers.he_normal(seed=12),  \n",
    "                 bias_initializer=initializers.he_uniform(seed=12),\n",
    "    filters=64, kernel_size=(3,3), padding='same', activation='linear'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(activations.relu))\n",
    "model.add(MaxPool2D(padding='same'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(256, activation='linear'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(activations.relu))\n",
    "model.add(Dense(84, activation='linear'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Activation(activations.relu))\n",
    "model.add(Dense(10, activation='linear'))\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples\n",
      "Epoch 1/2\n",
      "60000/60000 [==============================] - 62s 1ms/sample - loss: 0.3362 - accuracy: 0.8804\n",
      "Epoch 2/2\n",
      "60000/60000 [==============================] - 63s 1ms/sample - loss: 0.2162 - accuracy: 0.9204\n",
      "10000/1 - 2s - loss: 0.2302 - accuracy: 0.9119\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.24504419565200805, 0.9119]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, batch_size=100, epochs=2)\n",
    "\n",
    "model.evaluate(x_test,  y_test, batch_size=1000, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_json = model.to_json()\n",
    "with open(\"C:\\\\zaleslaw\\\\home\\\\models\\\\keras_experiments\\\\lenet_batchnorm\\\\modelConfig.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "    \n",
    "model.save_weights(\"C:\\\\zaleslaw\\\\home\\\\models\\\\keras_experiments\\\\lenet_batchnorm\\\\weights.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\zaleslaw\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\ops\\resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: C:\\zaleslaw\\home\\models\\keras_experiments\\lenet_batchnorm\\savedmodel\\assets\n"
     ]
    }
   ],
   "source": [
    "model.save(\"C:\\\\zaleslaw\\\\home\\\\models\\\\keras_experiments\\\\lenet_batchnorm\\\\savedmodel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
